package com.hyj.spark.CF


import com.alibaba.fastjson.{JSON, JSONArray}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.sql.functions._
import org.apache.spark.sql._
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema

import scala.collection.mutable


/**
  * sparksql-userCF 基于相似用户-获取推荐物品
  */
object UserBaseHyj {
  def main(args: Array[String]): Unit = {
    val spark: SparkSession = SparkSession.builder().master("local[*]")
      .appName("sparksql-userCF").enableHiveSupport()
      .getOrCreate()

    val df = spark.sql("select user_id,item_id,cast(rating as double) as rating  from badou.udata")

//        df.show()
    /**
      * |user_id|item_id|rating|
      * +-------+-------+------+
      * |    196|    242|     3|
      * |    186|    302|     3|
      * |     22|    377|     1|
      */

    /**
      * 1 计算用户相似度 这里使用cosine=a*b/(|a|*|b|)
      */
      //计算|a| |b|
    val u_df_abs = df.selectExpr("*", "pow(rating,2) as pow_rating").groupBy("user_id").agg(sum("pow_rating").as("pow_sum")) .selectExpr("user_id", "sqrt(pow_sum) as u_abs")
    //      .show()
    /**
      * -------+------------------+
      * |user_id|             u_abs|
      * +-------+------------------+
      * |    296|52.630789467763066|
      * |    467| 25.25866188063018|
      * |    691| 24.71841418861655|
      */
    val v_df_abs = u_df_abs.selectExpr("user_id as user_id_v","u_abs as u_abs_v")
    //计算 a*b
    val v_df = df.selectExpr("user_id as user_id_v","item_id","rating as rating_v")
    val dot_df = df.join(v_df, "item_id").filter("user_id!=user_id_v") .selectExpr("*", "(rating * rating_v) as dot").groupBy("user_id", "user_id_v").agg(sum("dot").as("dot_sum"))
    //      .show()
    /**
      * +-------+---------+--------+
      * |user_id|user_id_v|dot_sum|
      * +-------+---------+--------+
      * |    196|      617|    24.0|
      * |    196|      123|    71.0|
      * |    166|      206|   104.0|
      */

    val u_v_cos_df = dot_df.join(u_df_abs, "user_id").join(v_df_abs, "user_id_v") .selectExpr("user_id", "user_id_v", "dot_sum/(u_abs*u_abs_v) as cos")
//      .show()
    /**
      * +-------+---------+--------------------+
      * |user_id|user_id_v|                 cos|
      * +-------+---------+--------------------+
      * |    196|      617|0.033533914107330615|
      * |    196|      123| 0.10297059695164824|
      * |    166|      206| 0.29163935315828643|
      */

    //取top-N的相似用户
    import spark.implicits._
    val top_cos_df = u_v_cos_df.rdd.map(x => (x(0).toString, (x(1).toString, x(2).toString))).groupByKey()
      .mapValues(v => v.toArray.sortWith((v1, v2) => v1._2.toDouble.compareTo(v2._2.toDouble) > 0).slice(0, 5))

      /**
        * +-------+------------------------------------------------------------------------------------------------------------------------------------------+
        * |user_id|cos_v                                                                                                                                     |
        * +-------+------------------------------------------------------------------------------------------------------------------------------------------+
        * |385    |[[308, 0.49966117411717703], [833, 0.49490993620255647], [561, 0.4894204693024517], [747, 0.48914371760013464], [269, 0.4861011288308306]]|
        * |547    |[[111, 0.61757054089566], [820, 0.5468120033460924], [112, 0.5446384268877663], [284, 0.5271047230630211], [273, 0.5194974338578282]]     |
        * |709    |[[586, 0.5766019014551587], [37, 0.5530764637005143], [757, 0.5356752636320219], [222, 0.5196962518063857], [442, 0.5158462257733791]]    |
        */
      .flatMapValues(v => v) .toDF("user_id", "cos_v") .selectExpr("user_id", "cos_v._1 as user_id_v", "cos_v._2 as cos")
//      .show(false)
    /**
      * |user_id|user_id_v|cos                |
      * +-------+---------+-------------------+
      * |385    |308      |0.49966117411717703|
      * |385    |833      |0.49490993620255647|
      * |385    |561      |0.4894204693024517 |
      */

    /**
      * 2 根据相似用户计算推荐物品
      */

    val u_item_df = df.rdd.map(x => (x(0).toString, (x(1).toString, x(2).toString))).groupByKey() .mapValues(v => v.toArray).toDF("user_id", "arr_val")
//      .show(false)
    /**
      * |user_id|arr_val                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
      * +-------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |273    |[[328, 3.0], [345, 3.0], [311, 4.0], [902, 5.0], [900, 3.0], [690, 4.0], [319, 4.0], [321, 4.0], [303, 4.0], [305, 4.0], [315, 4.0], [316, 4.0], [340, 3.0], [272, 4.0], [347, 4.0], [313, 3.0], [304, 3.0], [307, 2.0], [286, 3.0], [268, 5.0], [896, 4.0], [338, 3.0]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
      * |528    |[[485, 2.0], [239, 5.0], [58, 5.0], [358, 2.0], [751, 4.0], [748, 3.0], [423, 1.0], [193, 4.0], [310, 3.0], [479, 4.0], [393, 2.0], [657, 5.0], [82, 4.0], [50, 5.0], [238, 3.0], [194, 5.0], [174, 5.0], [181, 5.0], [523, 4.0], [615, 4.0], [178, 4.0], [484, 3.0], [588, 2.0], [298, 4.0], [402, 4.0], [210, 5.0], [185, 4.0], [505, 4.0], [168, 4.0], [203, 4.0], [56, 3.0], [109, 4.0], [77, 3.0], [173, 5.0], [69, 3.0], [1618, 1.0], [83, 5.0], [294, 3.0], [541, 3.0], [31, 5.0], [427, 4.0], [258, 4.0], [845, 3.0], [204, 5.0], [526, 4.0], [422, 2.0], [202, 5.0], [250, 3.0], [79, 5.0], [1254, 3.0], [213, 4.0], [678, 3.0], [410, 4.0]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
      */

    val v_item_df = u_item_df.selectExpr("user_id as user_id_v","arr_val as arr_val_v")

    /**
      * 3 根据相似度和推荐物品计算物品打分
      */
      //关联用户喜欢的物品
    val score_df = top_cos_df.join(u_item_df, "user_id").join(v_item_df, "user_id_v")
      .selectExpr("user_id", "user_id_v", "cos", "arr_val", "arr_val_v")
    /**
      * +---------+-------+-------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |user_id_v|user_id|cos                |arr_val                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |arr_val_v                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
      * +---------+-------+-------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |296      |71     |0.33828954632615976|[[89, 5.0], [134, 3.0], [346, 4.0], [285, 3.0], [462, 5.0], [475, 5.0], [222, 3.0], [56, 5.0], [923, 5.0], [276, 4.0], [289, 2.0], [151, 1.0], [65, 5.0], [50, 3.0], [135, 4.0], [154, 3.0], [64, 4.0], [257, 5.0], [175, 4.0], [197, 5.0], [357, 5.0], [248, 3.0], [181, 3.0], [14, 5.0], [282, 3.0], [174, 2.0], [429, 4.0], [177, 2.0], [744, 4.0], [514, 4.0], [286, 4.0], [52, 4.0], [302, 3.0], [168, 5.0], [153, 4.0], [98, 4.0], [100, 4.0], [6, 3.0]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |[[705, 5.0], [508, 5.0], [20, 5.0], [228, 4.0], [222, 5.0], [429, 5.0], [855, 5.0], [248, 5.0], [258, 5.0], [242, 4.0], [48, 5.0], [286, 5.0], [272, 5.0], [510, 5.0], [275, 4.0], [427, 5.0], [83, 5.0], [961, 5.0], [544, 4.0], [32, 4.0], [1009, 3.0], [1073, 5.0], [96, 5.0], [292, 5.0], [950, 4.0], [1142, 5.0], [654, 5.0], [948, 1.0], [279, 4.0], [685, 4.0], [255, 2.0], [15, 3.0], [455, 1.0], [98, 5.0], [10, 2.0], [277, 5.0], [238, 4.0], [485, 5.0], [846, 2.0], [172, 5.0], [281, 2.0], [151, 2.0], [14, 4.0], [750, 5.0], [484, 4.0], [298, 1.0], [210, 3.0], [22, 4.0], [504, 5.0], [923, 5.0], [56, 5.0], [274, 4.0], [475, 4.0], [124, 5.0], [963, 5.0], [204, 5.0], [117, 3.0], [815, 3.0], [845, 5.0], [121, 5.0], [482, 5.0], [284, 4.0], [24, 2.0], [179, 4.0], [240, 1.0], [45, 5.0], [309, 1.0], [268, 4.0], [134, 5.0], [652, 4.0], [89, 5.0], [269, 5.0], [435, 5.0], [191, 5.0], [521, 4.0], [257, 5.0], [357, 5.0], [55, 5.0], [287, 4.0], [198, 5.0], [293, 5.0], [100, 5.0], [125, 5.0], [259, 1.0], [663, 5.0], [79, 4.0], [523, 4.0], [180, 5.0], [303, 4.0], [61, 3.0], [199, 5.0], [528, 5.0], [294, 1.0], [237, 5.0], [251, 5.0], [515, 5.0], [144, 4.0], [1160, 4.0], [282, 4.0], [1284, 4.0], [688, 1.0], [111, 3.0], [304, 3.0], [289, 3.0], [186, 3.0], [221, 5.0], [480, 5.0], [127, 5.0], [628, 5.0], [313, 5.0], [1007, 4.0], [50, 5.0], [256, 5.0], [19, 5.0], [297, 4.0], [301, 5.0], [315, 5.0], [1, 5.0], [7, 5.0], [181, 5.0], [483, 5.0], [11, 5.0], [114, 5.0], [285, 5.0], [211, 4.0], [276, 5.0], [9, 4.0], [13, 3.0], [244, 1.0], [632, 5.0], [194, 5.0], [469, 5.0], [250, 2.0], [153, 4.0], [898, 4.0], [150, 1.0], [246, 4.0], [514, 5.0], [659, 5.0], [462, 4.0], [696, 4.0], [1251, 5.0], [187, 5.0], [498, 5.0], [23, 5.0], [209, 4.0], [137, 4.0]]                                                                                                                                                                                                                                                                                                                                                                                                                                           |
      * |296      |753    |0.3968472894511972 |[[673, 1.0], [79, 4.0], [898, 4.0], [515, 5.0], [96, 1.0], [359, 4.0], [182, 3.0], [484, 5.0], [483, 5.0], [510, 4.0], [499, 3.0], [462, 4.0], [50, 4.0], [199, 5.0], [194, 4.0], [215, 5.0], [22, 4.0], [211, 4.0], [322, 3.0], [134, 4.0], [195, 1.0], [504, 3.0], [180, 2.0], [23, 2.0], [435, 4.0], [294, 5.0], [527, 4.0], [242, 4.0], [286, 3.0], [347, 2.0], [71, 5.0], [185, 3.0], [181, 3.0], [427, 5.0], [357, 4.0], [657, 5.0], [523, 4.0], [174, 4.0], [272, 4.0], [750, 2.0], [98, 5.0], [64, 4.0], [313, 5.0], [304, 4.0], [69, 4.0], [173, 5.0], [653, 4.0], [187, 3.0], [316, 4.0], [269, 5.0], [328, 3.0], [183, 1.0], [89, 3.0], [300, 1.0], [172, 3.0], [179, 2.0], [193, 4.0]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |[[705, 5.0], [508, 5.0], [20, 5.0], [228, 4.0], [222, 5.0], [429, 5.0], [855, 5.0], [248, 5.0], [258, 5.0], [242, 4.0], [48, 5.0], [286, 5.0], [272, 5.0], [510, 5.0], [275, 4.0], [427, 5.0], [83, 5.0], [961, 5.0], [544, 4.0], [32, 4.0], [1009, 3.0], [1073, 5.0], [96, 5.0], [292, 5.0], [950, 4.0], [1142, 5.0], [654, 5.0], [948, 1.0], [279, 4.0], [685, 4.0], [255, 2.0], [15, 3.0], [455, 1.0], [98, 5.0], [10, 2.0], [277, 5.0], [238, 4.0], [485, 5.0], [846, 2.0], [172, 5.0], [281, 2.0], [151, 2.0], [14, 4.0], [750, 5.0], [484, 4.0], [298, 1.0], [210, 3.0], [22, 4.0], [504, 5.0], [923, 5.0], [56, 5.0], [274, 4.0], [475, 4.0], [124, 5.0], [963, 5.0], [204, 5.0], [117, 3.0], [815, 3.0], [845, 5.0], [121, 5.0], [482, 5.0], [284, 4.0], [24, 2.0], [179, 4.0], [240, 1.0], [45, 5.0], [309, 1.0], [268, 4.0], [134, 5.0], [652, 4.0], [89, 5.0], [269, 5.0], [435, 5.0], [191, 5.0], [521, 4.0], [257, 5.0], [357, 5.0], [55, 5.0], [287, 4.0], [198, 5.0], [293, 5.0], [100, 5.0], [125, 5.0], [259, 1.0], [663, 5.0], [79, 4.0], [523, 4.0], [180, 5.0], [303, 4.0], [61, 3.0], [199, 5.0], [528, 5.0], [294, 1.0], [237, 5.0], [251, 5.0], [515, 5.0], [144, 4.0], [1160, 4.0], [282, 4.0], [1284, 4.0], [688, 1.0], [111, 3.0], [304, 3.0], [289, 3.0], [186, 3.0], [221, 5.0], [480, 5.0], [127, 5.0], [628, 5.0], [313, 5.0], [1007, 4.0], [50, 5.0], [256, 5.0], [19, 5.0], [297, 4.0], [301, 5.0], [315, 5.0], [1, 5.0], [7, 5.0], [181, 5.0], [483, 5.0], [11, 5.0], [114, 5.0], [285, 5.0], [211, 4.0], [276, 5.0], [9, 4.0], [13, 3.0], [244, 1.0], [632, 5.0], [194, 5.0], [469, 5.0], [250, 2.0], [153, 4.0], [898, 4.0], [150, 1.0], [246, 4.0], [514, 5.0], [659, 5.0], [462, 4.0], [696, 4.0], [1251, 5.0], [187, 5.0], [498, 5.0], [23, 5.0], [209, 4.0], [137, 4.0]]                                                                                                                                                                                                                                                                                                                                                                                                                                           |
      */
//    score_df.rdd.map(x=>{val arr_val = classOf[mutable.WrappedArray[String]].cast(x(3));val arr_val_v = classOf[mutable.WrappedArray[String]].cast(x(4)); (x(3).toString,x(4).toString)}).toDF().show()
    //筛选出用户u没接触过的物品和打分
    val u_v_score_rdd = score_df.rdd.map(x => {
      //      val arr_val = classOf[mutable.WrappedArray[Any]].cast(x(3))
      //      val arr_val_v = classOf[mutable.WrappedArray[Any]].cast(x(4))
      val array = x.getAs[mutable.WrappedArray[Any]](3)
      val array_v = x.getAs[mutable.WrappedArray[Any]]("arr_val_v")
      //      //用户u中的item去重
      val item_id_set = mutable.Set[String]()
      array.foreach(s => {
        //        val value: Any = s(0)
        val array: GenericRowWithSchema = s.asInstanceOf[GenericRowWithSchema]
        item_id_set.add(array.apply(0).toString)
      })
//    存储待推荐-item得分
      val target_item_map = mutable.Map[String, Double]()

      //过滤数据
      array_v.filter(arr => {
        val array: GenericRowWithSchema = arr.asInstanceOf[GenericRowWithSchema]
        !item_id_set.contains(array.apply(0).toString)
      })
        .foreach(arr => {
          val array: GenericRowWithSchema = arr.asInstanceOf[GenericRowWithSchema]
          //计算item得分=cos*rating
          val score: Option[Double] = target_item_map.get(array(0).toString)
          var d: Double = score.getOrElse(0)
          d += array.apply(1).toString.toDouble
//          target_item_map.put("rec_user_id:"+x.getAs[String]("user_id_v")+"|"+array(0).toString,
          target_item_map.put(array(0).toString,
            d * x.getAs[String]("cos").toString.toDouble)

        })

      (x.getAs[String]("user_id"), (x.getAs[String]("user_id_v"), target_item_map))
    })

//  .toDF("user_id", "user_id_v", "item_score")
    /**
      * +-------+---------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |user_id|user_id_v|item_score                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
      * +-------+---------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
      * |71     |296      |[[485, 1.6914477316307988], [204, 1.6914477316307988], [194, 1.6914477316307988], [272, 1.6914477316307988], [303, 1.353158185304639], [111, 1.0148686389784793], [210, 1.0148686389784793], [287, 1.353158185304639], [293, 1.6914477316307988], [1009, 1.0148686389784793], [45, 1.6914477316307988], [275, 1.353158185304639], [281, 0.6765790926523195], [269, 1.6914477316307988], [1284, 1.353158185304639], [659, 1.6914477316307988], [48, 1.6914477316307988], [950, 1.353158185304639], [845, 1.6914477316307988], [137, 1.353158185304639], [242, 1.353158185304639], [455, 0.33828954632615976], [632, 1.6914477316307988], [179, 1.353158185304639], [125, 1.6914477316307988], [515, 1.6914477316307988], [15, 1.0148686389784793], [251, 1.6914477316307988], [521, 1.353158185304639], [815, 1.0148686389784793], [24, 0.6765790926523195], [685, 1.353158185304639], [898, 1.353158185304639], [209, 1.353158185304639], [199, 1.6914477316307988], [277, 1.6914477316307988], [298, 0.33828954632615976], [688, 0.33828954632615976], [1251, 1.6914477316307988], [221, 1.6914477316307988], [484, 1.353158185304639], [83, 1.6914477316307988], [172, 1.6914477316307988], [7, 1.6914477316307988], [292, 1.6914477316307988], [187, 1.6914477316307988], [250, 0.6765790926523195], [259, 0.33828954632615976], [274, 1.353158185304639], [961, 1.6914477316307988], [268, 1.353158185304639], [1, 1.6914477316307988], [23, 1.6914477316307988], [544, 1.353158185304639], [652, 1.353158185304639], [121, 1.6914477316307988], [256, 1.6914477316307988], [469, 1.6914477316307988], [11, 1.6914477316307988], [427, 1.6914477316307988], [313, 1.6914477316307988], [32, 1.353158185304639], [1073, 1.6914477316307988], [1160, 1.353158185304639], [238, 1.353158185304639], [628, 1.6914477316307988], [696, 1.353158185304639], [508, 1.6914477316307988], [124, 1.6914477316307988], [498, 1.6914477316307988], [20, 1.6914477316307988], [301, 1.6914477316307988], [244, 0.33828954632615976], [79, 1.353158185304639], [211, 1.353158185304639], [180, 1.6914477316307988], [127, 1.6914477316307988], [523, 1.353158185304639], [1142, 1.6914477316307988], [294, 0.33828954632615976], [304, 1.0148686389784793], [198, 1.6914477316307988], [297, 1.353158185304639], [750, 1.6914477316307988], [963, 1.6914477316307988], [480, 1.6914477316307988], [279, 1.353158185304639], [1007, 1.353158185304639], [483, 1.6914477316307988], [150, 0.33828954632615976], [258, 1.6914477316307988], [55, 1.6914477316307988], [846, 0.6765790926523195], [186, 1.0148686389784793], [144, 1.353158185304639], [654, 1.6914477316307988], [435, 1.6914477316307988], [61, 1.0148686389784793], [9, 1.353158185304639], [663, 1.6914477316307988], [246, 1.353158185304639], [22, 1.353158185304639], [948, 0.33828954632615976], [855, 1.6914477316307988], [19, 1.6914477316307988], [255, 0.6765790926523195], [510, 1.6914477316307988], [10, 0.6765790926523195], [240, 0.33828954632615976], [504, 1.6914477316307988], [228, 1.353158185304639], [114, 1.6914477316307988], [315, 1.6914477316307988], [705, 1.6914477316307988], [191, 1.6914477316307988], [117, 1.0148686389784793], [13, 1.0148686389784793], [284, 1.353158185304639], [528, 1.6914477316307988], [96, 1.6914477316307988], [309, 0.33828954632615976], [482, 1.6914477316307988], [237, 1.6914477316307988]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
      * |753    |296      |[[485, 1.984236447255986], [204, 1.984236447255986], [303, 1.5873891578047887], [111, 1.1905418683535915], [210, 1.1905418683535915], [287, 1.5873891578047887], [293, 1.984236447255986], [1009, 1.1905418683535915], [248, 1.984236447255986], [45, 1.984236447255986], [275, 1.5873891578047887], [281, 0.7936945789023944], [1284, 1.5873891578047887], [659, 1.984236447255986], [48, 1.984236447255986], [257, 1.984236447255986], [950, 1.5873891578047887], [845, 1.984236447255986], [137, 1.5873891578047887], [455, 0.3968472894511972], [923, 1.984236447255986], [632, 1.984236447255986], [125, 1.984236447255986], [15, 1.1905418683535915], [251, 1.984236447255986], [521, 1.5873891578047887], [815, 1.1905418683535915], [24, 0.7936945789023944], [685, 1.5873891578047887], [209, 1.5873891578047887], [277, 1.984236447255986], [475, 1.5873891578047887], [298, 0.3968472894511972], [688, 0.3968472894511972], [1251, 1.984236447255986], [221, 1.984236447255986], [83, 1.984236447255986], [151, 0.7936945789023944], [56, 1.984236447255986], [7, 1.984236447255986], [292, 1.984236447255986], [250, 0.7936945789023944], [259, 0.3968472894511972], [274, 1.5873891578047887], [961, 1.984236447255986], [268, 1.5873891578047887], [289, 1.1905418683535915], [1, 1.984236447255986], [544, 1.5873891578047887], [652, 1.5873891578047887], [121, 1.984236447255986], [256, 1.984236447255986], [469, 1.984236447255986], [11, 1.984236447255986], [32, 1.5873891578047887], [1073, 1.984236447255986], [1160, 1.5873891578047887], [238, 1.5873891578047887], [14, 1.5873891578047887], [628, 1.984236447255986], [696, 1.5873891578047887], [514, 1.984236447255986], [508, 1.984236447255986], [124, 1.984236447255986], [498, 1.984236447255986], [20, 1.984236447255986], [301, 1.984236447255986], [244, 0.3968472894511972], [127, 1.984236447255986], [1142, 1.984236447255986], [198, 1.984236447255986], [297, 1.5873891578047887], [282, 1.5873891578047887], [276, 1.984236447255986], [963, 1.984236447255986], [480, 1.984236447255986], [100, 1.984236447255986], [279, 1.5873891578047887], [1007, 1.5873891578047887], [150, 0.3968472894511972], [258, 1.984236447255986], [55, 1.984236447255986], [285, 1.984236447255986], [846, 0.7936945789023944], [186, 1.1905418683535915], [144, 1.5873891578047887], [654, 1.984236447255986], [61, 1.1905418683535915], [663, 1.984236447255986], [9, 1.5873891578047887], [246, 1.5873891578047887], [153, 1.5873891578047887], [948, 0.3968472894511972], [855, 1.984236447255986], [19, 1.984236447255986], [255, 0.7936945789023944], [10, 0.7936945789023944], [240, 0.3968472894511972], [228, 1.5873891578047887], [114, 1.984236447255986], [315, 1.984236447255986], [705, 1.984236447255986], [191, 1.984236447255986], [222, 1.984236447255986], [117, 1.1905418683535915], [13, 1.1905418683535915], [429, 1.984236447255986], [284, 1.5873891578047887], [528, 1.984236447255986], [309, 0.3968472894511972], [482, 1.984236447255986], [237, 1.984236447255986]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
      * |376    |296      |[[485, 1.631760674890879], [204, 1.631760674890879], [194, 1.631760674890879], [272, 1.631760674890879], [303, 1.3054085399127033], [210, 0.9790564049345275], [287, 1.3054085399127033], [293, 1.631760674890879], [1009, 0.9790564049345275], [248, 1.631760674890879], [45, 1.631760674890879], [281, 0.6527042699563517], [1284, 1.3054085399127033], [659, 1.631760674890879], [48, 1.631760674890879], [257, 1.631760674890879], [950, 1.3054085399127033], [845, 1.631760674890879], [137, 1.3054085399127033], [242, 1.3054085399127033], [455, 0.32635213497817583], [923, 1.631760674890879], [632, 1.631760674890879], [179, 1.3054085399127033], [125, 1.631760674890879], [515, 1.631760674890879], [15, 0.9790564049345275], [251, 1.631760674890879], [521, 1.3054085399127033], [24, 0.6527042699563517], [685, 1.3054085399127033], [134, 1.631760674890879], [898, 1.3054085399127033], [89, 1.631760674890879], [209, 1.3054085399127033], [199, 1.631760674890879], [277, 1.631760674890879], [475, 1.3054085399127033], [298, 0.32635213497817583], [688, 0.32635213497817583], [1251, 1.631760674890879], [221, 1.631760674890879], [484, 1.3054085399127033], [83, 1.631760674890879], [151, 0.6527042699563517], [286, 1.631760674890879], [172, 1.631760674890879], [56, 1.631760674890879], [7, 1.631760674890879], [292, 1.631760674890879], [187, 1.631760674890879], [250, 0.6527042699563517], [259, 0.32635213497817583], [961, 1.631760674890879], [1, 1.631760674890879], [50, 1.631760674890879], [23, 1.631760674890879], [544, 1.3054085399127033], [652, 1.3054085399127033], [121, 1.631760674890879], [256, 1.631760674890879], [469, 1.631760674890879], [313, 1.631760674890879], [32, 1.3054085399127033], [1073, 1.631760674890879], [1160, 1.3054085399127033], [238, 1.3054085399127033], [628, 1.631760674890879], [696, 1.3054085399127033], [508, 1.631760674890879], [124, 1.631760674890879], [498, 1.631760674890879], [20, 1.631760674890879], [244, 0.32635213497817583], [79, 1.3054085399127033], [211, 1.3054085399127033], [180, 1.631760674890879], [127, 1.631760674890879], [523, 1.3054085399127033], [1142, 1.631760674890879], [294, 0.32635213497817583], [304, 0.9790564049345275], [297, 1.3054085399127033], [282, 1.3054085399127033], [276, 1.631760674890879], [750, 1.631760674890879], [963, 1.631760674890879], [480, 1.631760674890879], [279, 1.3054085399127033], [1007, 1.3054085399127033], [483, 1.631760674890879], [150, 0.32635213497817583], [258, 1.631760674890879], [55, 1.631760674890879], [462, 1.3054085399127033], [285, 1.631760674890879], [846, 0.6527042699563517], [186, 0.9790564049345275], [144, 1.3054085399127033], [654, 1.631760674890879], [435, 1.631760674890879], [61, 0.9790564049345275], [9, 1.3054085399127033], [153, 1.3054085399127033], [22, 1.3054085399127033], [948, 0.32635213497817583], [855, 1.631760674890879], [19, 1.631760674890879], [255, 0.6527042699563517], [510, 1.631760674890879], [10, 0.6527042699563517], [240, 0.32635213497817583], [504, 1.631760674890879], [228, 1.3054085399127033], [114, 1.631760674890879], [315, 1.631760674890879], [191, 1.631760674890879], [222, 1.631760674890879], [117, 0.9790564049345275], [13, 0.9790564049345275], [429, 1.631760674890879], [284, 1.3054085399127033], [528, 1.631760674890879], [96, 1.631760674890879], [309, 0.32635213497817583], [482, 1.631760674890879]]                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
      */

    //根据user_id分组，相同的item打分相加，取top-N item
    val u_rec_item_rdd: RDD[(String, List[(String, Double)])] = u_v_score_rdd.groupByKey().mapValues(v => {
      val res_map = mutable.Map[String, Double]()
      //得到所有的打分map
      val arr: Array[(String, mutable.Map[String, Double])] = v.toArray
      arr.foreach(t => {
        t._2.foreach(item => {
          var score: Double = res_map.getOrElse(item._1, 0)
          //相同item打分相加
          var value: Double = item._2
          res_map.put(item._1, score+value)
        })
      })
      //根据打分排序 取top-N
      res_map.toList.sortWith((t1, t2: (String, Double)) => t1._2.compareTo(t2._2) > 0).take(10)
    })

    u_rec_item_rdd.toDF().show(false)



    /**
      *  推荐user_id=71的物品
      */
    val rec_df: DataFrame = u_rec_item_rdd.toDF("user_id", "rec_item")
    rec_df
      .where("user_id=385")
      .select(rec_df("user_id"),explode(rec_df("rec_item")))
      .toDF("user_id","rec_item")
      .selectExpr("user_id","rec_item._1 as item_id","rec_item._2 as score")
      .show(false)

    /**
      * +-------+-------+------------------+
      * |user_id|item_id|score             |
      * +-------+-------+------------------+
      * |385    |11     |10.83455940330164 |
      * |385    |179    |10.820999358015293|
      * |385    |432    |10.815509891115187|
      * |385    |475    |9.827860145138558 |
      * |385    |154    |9.356610793984192 |
      * |385    |7      |9.345355108481666 |
      * |385    |64     |9.341759016307728 |
      * |385    |223    |8.381684906489413 |
      * |385    |202    |8.375322977248489 |
      * |385    |640    |8.360982495742789 |
      * +-------+-------+------------------+
      */

  }

}
